package com.qen.yanxuan

import cn.hutool.core.date.DateUtil
import cn.hutool.core.io.file.FileWriter
import cn.hutool.core.lang.Console
import cn.hutool.core.util.StrUtil
import cn.hutool.poi.excel.ExcelUtil
import com.qen.yanshen.THREAD_COUNT
import com.qen.yanshen.UserAgent
import it.skrape.core.htmlDocument
import it.skrape.fetcher.HttpFetcher
import it.skrape.fetcher.Method
import it.skrape.fetcher.response
import it.skrape.fetcher.skrape
import org.apache.commons.collections4.MapUtils
import java.util.Date
import java.util.concurrent.atomic.AtomicInteger
import kotlin.collections.forEach

const val YanxuanExcelPath = "/Users/a1/Documents/Git/Kotlin/Eleven/out/yanxuan/20241023"

fun mainx() {

    val reader = ExcelUtil.getReader(YanxuanExcelPath + "/Yanxuan-20241023.xlsx")
    val all = reader.readAll()
    val maxNum = all.size

    /**爬虫线程队列*/
    val threads = ArrayList<Thread>()

    /**已爬取的新闻数（所谓“已爬取”，实际是从拿到链接就算）*/
    val pageNum = AtomicInteger()

    /**爬取结束信号*/
//    val newsTargetReachedSemaphore = CountDownLatch(THREAD_COUNT)
    var referer = YanXuanUrl
    for (i in 0..THREAD_COUNT) {
        val thread = Thread {
            try {
                while (true) {
                    val index = pageNum.incrementAndGet()
                    if (index >= maxNum) {
                        break
                    }
                    val obj = all.get(index)
                    Console.log("${Thread.currentThread().threadId()}：pageNum:${index}")
                    try {
                        contentUrl(MapUtils.getString(obj, "url"), MapUtils.getString(obj, "title"))
                    } catch (e: Exception) {
                        e.printStackTrace()
                    }

                    referer = MapUtils.getString(obj, "url")
                }
            } catch (e: Exception) {
                e.printStackTrace()
            }
        }
        thread.start()
        threads.add(thread)
    }
    threads.forEach { t -> t.join() }
    Console.error("多线程结束了")
}

fun contentUrl(goUrl: String, title: String) {
    val content = StringBuffer()
    skrape(HttpFetcher) {
        request {
            url = goUrl
            headers = initYanXuanHeader()
            cookies = initYanXuanCookie()
            userAgent = UserAgent
            method = Method.GET
            body = null
            timeout = 60000
//            proxy = ProxyBuilder(Proxy.Type.HTTP, "127.0.0.1", 26001)
        }
        response {
            htmlDocument {
                findFirst(".entry-content").children.forEach { t ->
                    if (t.tagName == "p") {
                        content.append(t.text + "\n")
                    }
                    if (t.classNames.contains("typology-link-pages")) {
                        t.children.forEach { t ->
                            val href = t.attribute("href")
                            if (StrUtil.isNotBlank(href)) {
                                content.append(contentUrlPage(href))
                            }
                        }
                    }
                }
            }
        }
    }
    val writer = FileWriter(
        YanxuanOutFile + "/${
            DateUtil.format(
                Date(),
                "YYYYMMdd"
            )
        }/txt/$title-${System.currentTimeMillis()}.txt"
    )
    writer.write(content.toString())
}


fun contentUrlPage(goUrl: String): String {
    val doc = StringBuffer()
    skrape(HttpFetcher) {
        request {
            url = goUrl
            headers = initYanXuanHeader()
            cookies = initYanXuanCookie()
            userAgent = UserAgent
            method = Method.GET
            body = null
//            proxy = ProxyBuilder(Proxy.Type.HTTP, "127.0.0.1", 26001)
        }
        response {
            htmlDocument {
                findFirst(".entry-content").children.forEach { t ->
                    if (t.tagName == "p") {
                        doc.append(t.text + "\n")
                    }
                }
            }
        }
    }
    return doc.toString();
}

fun main() {
    contentUrl(
        "https://www.yanxuanwk.com/%e6%88%91%e5%92%8c%e8%80%81%e5%85%ac%e4%ba%92%e6%8d%a2%e8%ba%ab%e4%bd%93/",
        "我和老公互换身体"
    )
    contentUrlPage("https://www.yanxuanwk.com/%e6%88%91%e5%92%8c%e8%80%81%e5%85%ac%e4%ba%92%e6%8d%a2%e8%ba%ab%e4%bd%93/2/")
}

fun maint() {
    val threads = ArrayList<Thread>()
    val pageNum = AtomicInteger()

    for (i in 0..THREAD_COUNT) {
        val thread = Thread {
            while (true) {
                Console.log("${Thread.currentThread().threadId()}：pageNum:${pageNum.get()}")
                val x = pageNum.incrementAndGet()
                if (x >= 100) {
                    break
                }
            }
        }
        thread.start()
        threads.add(thread)
    }
    threads.forEach { t -> t.join() }
}